/**
 * 
 */
package com.spider.silence.worker;

import java.net.URLDecoder;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.TianYaModuleTask;
import com.spider.silence.task.TianYaTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.PropertiesUtil;

import us.codecraft.webmagic.Page;

/**
 * @title TianYaModuleWorker.java
 * @author liyuanguo
 * @time 2017年1月9日 上午10:13:36
 * @description
 * @version V1.0
 */
public class TianYaModuleWorker  extends AbstractCommonDownLoader implements Runnable{
	private Logger logger =Logger.getLogger(TianYaModuleWorker.class);
	@Override
	public void run() {
		while(true&&!Thread.interrupted()){
			try {
				TianYaModuleTask moduleTask=(TianYaModuleTask) CommonTaskManager.takeTask("tianYaModule",2000);
				if(null!=moduleTask){
					String moduleUrl=moduleTask.getModuleUrl();
//					int moduleNum=moduleTask.getModuleNum();
//					int moduleIndex=moduleTask.getModuleindex();
					//Document document = get(moduleUrl,true);
					boolean flag=true;
					long startTime=System.currentTimeMillis();
					long nowTime=startTime;
					while(flag&&nowTime-startTime<Long.valueOf(PropertiesUtil.getVal("TianYa.TimeOut"))){
						try {
							//doc= get("http://www.mala.cn/forum.php",true);
							Document document=null;
							Page pageContent=null;//ProxyPool.proxy()
							pageContent= getHtml(moduleUrl, true);
							if(null==pageContent){

							}else if(pageContent.getStatusCode()==0){

							}else if(String.valueOf(pageContent.getStatusCode()).startsWith("1")){

							}else if(pageContent.getStatusCode()==200){
								flag=false;
								//System.out.println("爬取模块下帖子url:"+moduleUrl+"状态码："+pageContent.getStatusCode());
								document=pageContent.getHtml().getDocument();
								//获取每个模块下的第一页所有帖子的url、组装任务
								Elements elements=document.select("tbody:gt(1)").select(".td-title").select("a");
								for(int j=0;j<elements.size();j++){
									String forumUrl=elements.get(j).absUrl("href");
									TianYaTask task =new TianYaTask(URLDecoder.decode(forumUrl,"UTF-8"));
									CommonTaskManager.add("tianYa",task);
								}
//								//历史数据爬取start
//								Element element=document.select(".short-pages-2.clearfix").get(0); //页码的数量
//								Elements pages=element.select("a");
//								if(pages.size()>1) {
//									if ( pages.get(1).text().equals("下一页")) {
//										String url = pages.get(1).attr("href");
//										ExecutorService executorService = Executors.newFixedThreadPool(20);
//										CramlHistory cramlHistory=new CramlHistory(url);
//										executorService.submit(cramlHistory);
//									}
//								}
//								//历史数据爬取end
							}else if(String.valueOf(pageContent.getStatusCode()).startsWith("3")){

							}else if(String.valueOf(pageContent.getStatusCode()).startsWith("4")){

							}else if(String.valueOf(pageContent.getStatusCode()).startsWith("5")){

							}else{

							}

							nowTime=System.currentTimeMillis();
						} catch (Exception e1) {
							//System.out.println("异常信息："+e1.getMessage());
							System.out.println(moduleUrl);
							logger.error(e1.getMessage(),e1);
						}
					}


				}


			} catch (Exception e) {
				logger.error(e.getMessage(),e);
				continue;
			}
		}
		
	}

	/**
	 * 爬取历史帖子id
	 */

	public class CramlHistory implements  Runnable{
		int pageNo=2;
		private String url;
		@Override
		public void run() {
			cramlHistoryData(url);

		}
		public CramlHistory(String url){
			this.url=url;
		}

		public void cramlHistoryData(String url){
			pageNo++;
			Elements pages=null;
			Document document=null;
				try {
					Page pageContent=null;//ProxyPool.proxy()
					pageContent= getHtml(url, ProxyPool.proxy());
					if (null == pageContent) {

					} else if (pageContent.getStatusCode() == 0) {

					} else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

					} else if (pageContent.getStatusCode() == 200) {
						document = pageContent.getHtml().getDocument();
						Element element=document.select(".short-pages-2.clearfix").get(0); //页码的数量
						pages=element.select("a");
						Elements elements = document.select("tbody:gt(1)").select(".td-title").select("a");
						for (int j = 0; j < elements.size(); j++) {
							String forumUrl = elements.get(j).absUrl("href");
							TianYaTask task = new TianYaTask(URLDecoder.decode(forumUrl, "UTF-8"));
							CommonTaskManager.add("tianYa", task);
							System.out.println(document.title()+"模块的第"+pageNo+"页的第"+j+"个帖子的url："+forumUrl);
						}
					}
				}catch (Exception e) {
					e.printStackTrace();
				}finally {
					System.out.println(document.title()+"模块的第"+pageNo+"页数据爬取完毕。。。");
				}
			if(pages.size()==3){
				String next_url=pages.get(2).attr("href");
				cramlHistoryData(next_url);
			}
		}
	}
}
